<?php
/**
*   copyright:深圳市亿游科技开发有限公司
 *  v1.1
 */
define('MAX_LINES',200000);//最多读取多少行
define('MAX_FETCH_SIZE',100000000);//最多读取多少行
define('DEFAULT_PAGE_TIME',200);

class myspider{
	public static $init = false;
	public static $ch;
	public static $url;
	public static $result;
	public static $headers;
	public static $curlInfo;
	public static $agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)";
	public static $cookie = "";
	public static $referer = "";
	public static $isJson = false;
	public static $proxy = '';
	public static $cookieEnabled = false;
	public static $timeout = DEFAULT_PAGE_TIME;
	public static $accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';

	public static function openFile($url){
			self::$result = file_get_contents($url);//发送HTTPs请求并获取返回的数据，推荐使用curl
			return self::$result;
	}

	public static function prepareHeaders(){
        self::$headers = array(
            'ACCEPT_ENCODING:gzip,deflate'
        ,'ACCEPT:'.self::$accept
        ,'Connection:keep-alive'
        );
	}

    public static function setNoCache(){
        curl_setopt(self::$ch, CURLOPT_FRESH_CONNECT, TRUE);
    }

	public static function setJsonAccept($isJson = true){
		self::$isJson = $isJson;
	}

	public  static function  setAgent($agent){
		self::$agent = $agent;
	}
	
	public  static function  setCookie($cookie){
		self::$cookie = $cookie;
	}

	public static function setTimeout($timeout){
		self::$timeout = $timeout;
	}

	public static function post($url, $data=[]){
		self::$url      = $url;
		if(!is_array($data)){
			$tmp = json_decode($data);
			if(!$tmp && $data){
				echo 'error:not correct  ajax format ';
				print_r($data);
				die();
			}
			$data = $tmp;
		}
		self::prepare($url);
		curl_setopt(self::$ch, CURLOPT_POST, 1);//设置为POST方式
		curl_setopt(self::$ch, CURLOPT_POSTFIELDS, http_build_query($data));
		curl_setopt(self::$ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt(self::$ch, CURLOPT_USERAGENT, self::$agent);
		curl_setopt(self::$ch, CURLOPT_COOKIE , self::$cookie);
        curl_setopt(self::$ch, CURLOPT_HTTPHEADER,self::$headers);
		self::$result = curl_exec(self::$ch);
		self::$curlInfo = curl_getinfo(self::$ch);
		self::$headers  = self::joinArrayKeyValue(self::$curlInfo);
		curl_close(self::$ch);
		return self::$result;
	}

	//join 二维数组
	public static function joinArrayKeyValue($arr,$sep=':'){
		$str = '';
		foreach($arr as $key=>$val){
			$str .= $key.$sep.$val."\r\n";
		}
		return $str;
	}

	//put ajax to php://input
	public static function ajax($url,$data=[]){
		self::$isJson = true;
		self::prepare($url);
		if (!empty($data)) {
			if(is_array($data)){
				$data = json_encode( $data,JSON_UNESCAPED_UNICODE);
			}
			curl_setopt(self::$ch, CURLOPT_POST, 1);//设置为POST方式
			curl_setopt(self::$ch,CURLOPT_POSTFIELDS,$data);
            $ajaxHeaders = array('Content-Type: application/json; charset=utf-8','Content-Length:' . strlen($data));
            self::$headers = array_merge(self::$headers,$ajaxHeaders);
		}
		curl_setopt(self::$ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt(self::$ch, CURLOPT_HTTPHEADER,self::$headers);
		self::$result = curl_exec(self::$ch);
		self::$curlInfo = curl_getinfo(self::$ch);
		self::$headers  = self::joinArrayKeyValue(self::$curlInfo);
		curl_close(self::$ch);
		return self::$result;
	}

	public static function prepare($url){
		self::$url      = $url;
		self::$ch = curl_init();
		self::prepareProxy();
		self::prepareCookie();
		self::prepareAccept();
		self::prepareReferer();
		self::prepareHeaders();
		self::setNoCache();
		curl_setopt(self::$ch, CURLOPT_USERAGENT, self::$agent);
		curl_setopt(self::$ch, CURLOPT_FOLLOWLOCATION,1);
		curl_setopt(self::$ch, CURLOPT_TIMEOUT, self::$timeout);
		//curl_setopt(self::$ch, CURLOPT_MAXREDIRS,5);
		curl_setopt(self::$ch, CURLOPT_RETURNTRANSFER, true);
		//curl_setopt(self::$ch, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
		curl_setopt(self::$ch,CURLOPT_HEADER,0);
		curl_setopt(self::$ch, CURLOPT_SSL_VERIFYPEER, false);
		curl_setopt(self::$ch, CURLOPT_SSL_VERIFYHOST, false);
		curl_setopt(self::$ch, CURLOPT_URL, self::$url);
	}

	public static function prepareAccept(){
		if(self::$isJson){
			self::$accept = 'application/json';
		}else{
			self::$accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
		}
	}

	public static function prepareCookie(){
		if(self::$cookieEnabled){
			$cookieFile = "cookieFile".self::$proxy;
			curl_setopt(self::$ch, CURLOPT_COOKIEJAR, $cookieFile);
			curl_setopt(self::$ch, CURLOPT_COOKIEFILE, $cookieFile);
			curl_setopt(self::$ch, CURLOPT_COOKIE , self::$cookie);
		}
	}

	public static function setCookieEnabled($useCookie=true){
		self::$cookieEnabled = $useCookie;
	}

	public static function prepareProxy(){
		if("" <> self::$proxy){
			curl_setopt(self::$ch,CURLOPT_PROXY,self::$proxy);
		}
	}

	public static function setProxy($proxy=''){
		self::$proxy = $proxy;
	}

	public static function setReferer($referer){
		self::$referer = $referer;
	}

	public static function prepareReferer(){
		if("" == self::$referer){
			$referInfo = parse_url(self::$url);
			self::$referer = $referInfo['scheme']."://".$referInfo['host'];
		}
		curl_setopt(self::$ch, CURLOPT_REFERER,self::$referer);
	}

	public  static  function get($url) {
		self::prepare($url);
        curl_setopt(self::$ch, CURLOPT_HTTPHEADER,self::$headers);
		self::$result = curl_exec (self::$ch);
		self::$curlInfo = curl_getinfo(self::$ch);
		self::$headers  = self::joinArrayKeyValue(self::$curlInfo);

		curl_close (self::$ch);
		if(intval(self::$curlInfo['http_code']) >= 400){
			self::$result = ['code'=>self::$curlInfo['http_code']];
		}
		
		return self::$result;		
	}
	
	public static function getContentLink($newUrl,$baseUrl=""){
		$returnUrl = '';
		if(!$baseUrl){
			$baseUrl   = self::$curlInfo['url'];
		}
		$info = self::parseUrl($baseUrl);
		if(!(preg_match("/^http:/i",$newUrl) || preg_match("/^https:/i",$newUrl))){//非http://开头
			if(preg_match('{^/}i',$newUrl)){//以/开头
				$returnUrl = $info['scheme'].'://'.$info['host'].$newUrl;
			}else if(preg_match('{^\.+}i',$newUrl)){//以.开头
				$dirs  = preg_split('{/+}is',$info['dir'],-1,PREG_SPLIT_NO_EMPTY);
				$files = preg_split('{\.[/]+}',$newUrl,-1);
				$skipDir = 0;
				for($i = 0; $i < count($files); $i++){
					if($files[$i]=='.'){
						$skipDir++;
					}else if($files[$i]==''){//继续执行./
						continue;
					}else{//得到最后链接并跳出
						$tailUrl = $files[$i];
						break;
					}
				}
				
				$headUrl = '';
				for($i =0; $i < (count($dirs)-$skipDir); $i++){
					$headUrl .= "/".$dirs[$i];
				}
			
				$returnUrl = $info['scheme'].'://'.$info['host'].$headUrl.'/'.$tailUrl;
			}else{//以image类似开头
				$returnUrl = $info['scheme'].'://'.$info['host'].$info['dir'].$newUrl;
			}
		}else{//http://开头
			$returnUrl = $newUrl;
		} 
	
		return $returnUrl;	
	}
	
	public static function parseUrl($url) {
		$r  = "^(?:(?P<scheme>\w+)://)?";
		$r .= "(?:(?P<login>\w+):(?P<pass>\w+)@)?";
		$r .= "(?P<host>(?:(?P<subdomain>[\w\.]+)\.)?" . "(?P<domain>\w+\.(?P<extension>\w+)))";
		$r .= "(?::(?P<port>\d+))?";
		$r .= "(?P<path>[\w/]*/(?P<file>\w+(?:\.\w+)?)?)?";
		$r .= "(?:\?(?P<arg>[\w=&]+))?";
		$r .= "(?:#(?P<anchor>\w+))?";
		$r = "!$r!";                                                // Delimiters
    
		preg_match ( $r, $url, $out );
    
		if(isset($out['file'])){
			$out['dir'] = substr($out['path'],0,-strlen($out['file'])).'/';
		}else{
			$out['dir'] = $out['path'].'/';
		}
	
		$out['dir'] = preg_replace("{/+$}","/",$out['dir']);
	
		return $out;
	}

	public static function open($url){
		$fp = fopen($url,'r');

		if(!$fp){
			return false;
		}


		$html = '';
		$i = 0;
		while (!feof($fp)) {
			$i++;
			if($i > MAX_LINES) break;
			$html .= fgets($fp,1024);
		}

		fclose($fp);
		self::$result = $html;
		return $html;
	}

	public static function fetch($url){
		return self::get($url);
	}
	
	public static function utf8(){
		return self::utf8Result();
	}
	
	public static function utf8Result(){
		self::$charset = self::getCharset();
		if(preg_match("{^utf}i",self::$charset)){
			return self::$result;
		}else{
			return self::iconv(self::$charset,"utf-8");
		}
	}
	
	public static function getCharset(){
		$find = preg_match("{charset=(\S+)}is",self::$headers,$matches);
		if($find){
			return $matches[1];
		}else{
		   $reg = "{<meta.*?charset=([\w\-]+)}i";
		   $find = preg_match($reg,self::$result,$matches);
		   if($find){
			return $matches[1];
		   }else{
			return 'utf-8';//默认语言
		   }
		}		
	}
	
	public static function getHeaders(){
		return self::headers;
	}

	public static function gbkToUtf8(){
		return self::iconv("gbk","utf-8");
	}

	public static  function gb2312ToUtf8(){
		return self::iconv("gb2312","utf-8");
	}

	public static function iconv($srcLang,$targetLang,$content=""){
		if(!$content){$content = self::$result;}
		return @iconv($srcLang,$targetLang.'//IGNORE',$content);
	}

	public static function result(){
		return self::$result;
	}

}